package test2.spider.news.cnfol;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import jodd.http.HttpRequest;
import jodd.http.HttpResponse;
import test2.spider.BaseProcessor;

public class MainProcessor extends BaseProcessor {

	@Override
	public void process(String url) {
		System.out.println("发送http请求到--->" + url);
		HttpResponse response = HttpRequest.get(url).timeout(10000).send();
		int statusCode = response.statusCode();
		if(statusCode==404) {
			throw new RuntimeException("the end");
		}
		String html = response.charset("utf-8").bodyText();
		if(html!=null) {
			Document doc = Jsoup.parse(html);
			Elements elements = doc.select(".NewsLstItem");
			for(Element element : elements) {
				Elements lis = element.select("li");
				for(Element e : lis) {
					Element a = e.select("a").get(0);
					
					String href = a.attr("href");
					Cnfol.threadPool.submit(new Runnable() {
						@Override
						public void run() {
							new DetailProcessor().process(href);
						}
					});
				}
			}
		}
	}

}
